package net.net23.sysdent.server.jdo.control.search.model;

import java.io.IOException;
import java.io.StringReader;
import java.util.HashSet;
import java.util.Set;
import java.util.logging.Logger;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.snowball.SnowballAnalyzer;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;

public class SearchJanitorUtils {

	private static final Logger log = Logger.getLogger(SearchJanitorUtils.class
			.getName());

	public final static Set<String> stopWords = new HashSet<String>();
	static {
		stopWords.add("a");
		stopWords.add("an");
		stopWords.add("and");
		stopWords.add("are");
		stopWords.add("as");
		stopWords.add("at");
		stopWords.add("be");
		stopWords.add("by");
		stopWords.add("for");
		stopWords.add("if");
		stopWords.add("in");
		stopWords.add("into");
		stopWords.add("is");
		stopWords.add("no");
		stopWords.add("not");
		stopWords.add("of");
		stopWords.add("on");
		stopWords.add("or");
		stopWords.add("such");
		stopWords.add("that");
		stopWords.add("the");

		stopWords.add("their");
		stopWords.add("then");
		stopWords.add("there");
		stopWords.add("these");
		stopWords.add("they");
		stopWords.add("this");
		stopWords.add("to");
		stopWords.add("was");
		stopWords.add("will");
		stopWords.add("with");
	}

	public static Set<String> getTokensForIndexingOrQuery(String index_raw,
			int maximumNumberOfTokensToReturn) {
		String indexCleanedOfHTMLTags = index_raw.replaceAll("\\<.*?>", " ");
		Set<String> returnSet = new HashSet<String>();
		try {
			Analyzer analyzer = new SnowballAnalyzer(
					org.apache.lucene.util.Version.LUCENE_30, "English",
					stopWords);
			TokenStream tokenStream = analyzer.tokenStream("content",
					new StringReader(indexCleanedOfHTMLTags));
			tokenStream.reset();
			while (tokenStream.incrementToken()
					&& (returnSet.size() < maximumNumberOfTokensToReturn)) {
				if( tokenStream.hasAttribute( TermAttribute.class ) ) {
			        TermAttribute attr = tokenStream.getAttribute( TermAttribute.class );
			        returnSet.add( attr.term() );
			      }
			}
		} catch (IOException e) {
			log.severe(e.getMessage());
		}
		return returnSet;
	}
}
